#!/usr/bin/env python
# encoding: utf-8

"""
@version: 0.1
@author: Kouen
@license: Apache Licence
@email: jobkouen@outlook.com
@software: PyCharm Community Edition
@file: ow_daily.py
@time: 2016/11/3 上午11:28
@简介:

"""
import os, sys
sys.path.insert(0, os.getcwd()[:os.getcwd().find('streaming')-1])

from functools import reduce
from scrapy.contrib.spiders import Spider
import scrapy
import re
import SimpleMonitorConfig as INI


class StreamingItem(scrapy.Item):
    app_name = scrapy.Field()
    app_id = scrapy.Field()
    active = scrapy.Field()
    completed = scrapy.Field()


class ScrapyStreamApp(Spider):
    name = "ScrapyStreamApp"

    def start_requests(self):
        for url in INI.AppListURL:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        # 获取列表内容
        content = response.xpath('//script/text()')[-1].extract()
        part = re.compile(r'href=\'/cluster/app/(app.*?)\'.*"SPARK","(root.*?)"', re.I | re.M)
        for app, status in part.findall(content):
            if 'root.streaming' in status:
                for url in INI.SteamDetailURL:
                    yield scrapy.Request(url=url.format(App=app), callback=self.parse_steaming)
            elif 'root.day' in status:
                pass

    def parse_steaming(self, response):
        item = StreamingItem()
        part = re.compile(r'/proxy/(app.*?)/', re.I | re.M)

        item['app_name'] = response.xpath("//strong")[0].xpath("text()").extract()[0]
        item['app_id'] = part.findall(response.url)[0]
        item['active'] = list()
        item['completed'] = list()

        def _get_info_(_key, _num, _f):
            if not _num:
                l = response.xpath('//table[@id="%s-batches-table"]/descendant::tr' % _key)
            else:
                l = response.xpath('//table[@id="%s-batches-table"]/descendant::tr' % _key)[:_num]
            for td in l:
                schedule = reduce(
                    lambda x, y: int(x/y*100) if x < y else int(y/x*100),
                    map(
                        lambda x: float(x),
                        td.xpath("descendant::span/text()").extract()[0].strip().split('/')
                    )
                )
                name = td.xpath("child::td/@id").extract()[0]
                status = _f(schedule)
                event = int(td.xpath("child::td")[1].xpath("@sorttable_customkey").extract()[0])

                if "completed" is _key:
                    t = td.xpath("child::td")[3].xpath("text()").extract()[0].strip().replace(" ", '')
                    run_time = float(t.replace("s", "")) if "s" in t else float(t.replace("min", ""))*60
                else:
                    run_time = 0

                item[_key].append((name, status, schedule, int(name.replace("batch-", "")[:-3]), event, run_time))
        _get_info_("active", 0, lambda x: "Process" if x else "Suspend")
        _get_info_("completed", INI.CheckEventListCount+1, lambda x: "Completed")
        return item

